{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "\n",
    "os.environ['CUDA_VISIBLE_DEVICES'] = '1'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "from transformers import T5Tokenizer, T5ForConditionalGeneration\n",
    "\n",
    "tokenizer = T5Tokenizer.from_pretrained('mesolitica/t5-small-standard-bahasa-cased')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['finetune-t5-base-standard-bahasa-cased-combined/checkpoint-640000',\n",
       " 'finetune-t5-base-standard-bahasa-cased-combined/checkpoint-650000',\n",
       " 'finetune-t5-base-standard-bahasa-cased-combined/checkpoint-660000']"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from glob import glob\n",
    "\n",
    "checkpoints = sorted(glob('finetune-t5-base-standard-bahasa-cased-combined/checkpoint-*'))\n",
    "checkpoints"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = T5ForConditionalGeneration.from_pretrained(checkpoints[-1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# model2 = T5ForConditionalGeneration.from_pretrained('mesolitica/finetune-translation-t5-small-standard-bahasa-cased')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<pad> Hi guys! I noticed yesterday & today many people have got these cookies, right? So today I want to share some post mortem of our first batch:</s>\n"
     ]
    }
   ],
   "source": [
    "input_ids = tokenizer.encode('terjemah Melayu ke Inggeris: Hi guys! I noticed semalam & harini dah ramai yang dapat cookies ni kan. So harini i nak share some post mortem of our first batch:', return_tensors = 'pt')\n",
    "outputs = model.generate(input_ids, max_length = 100)\n",
    "print(tokenizer.decode(outputs[0]))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<pad> Can you solve it?</s>\n"
     ]
    }
   ],
   "source": [
    "input_ids = tokenizer.encode('terjemah Melayu ke Inggeris: mesolitica boleh buat asr tak', return_tensors = 'pt')\n",
    "outputs = model.generate(input_ids, max_length = 100)\n",
    "print(tokenizer.decode(outputs[0]))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<pad> Brother, I want to copy it on Facebook.. haha. If you can't, I'll</s>\n"
     ]
    }
   ],
   "source": [
    "input_ids = tokenizer.encode('terjemah Melayu ke Inggeris: Bang, aku nak copy masuk kat fb…hahaha. Kalau xleh aku ss', return_tensors = 'pt')\n",
    "outputs = model.generate(input_ids, max_length = 100)\n",
    "print(tokenizer.decode(outputs[0]))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<pad> Sesungguhnya. Ini bukan pakar, saya tahu. Ia isyarat, bodoh.</s>\n"
     ]
    }
   ],
   "source": [
    "input_ids = tokenizer.encode(\"terjemah pasar Melayu ke Melayu: Memanglah. Ini tak payah expert, aku pun tau. It's a gesture, bodoh.\", return_tensors = 'pt')\n",
    "outputs = model.generate(input_ids, max_length = 100)\n",
    "print(tokenizer.decode(outputs[0]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['<pad> rakyat mmg x suka ko pun</s> <pad>', '<pad> org melaka pun tak suka kau</s>', '<pad> rakyat memang x suka kau pun</s> <pad> <pad>']\n"
     ]
    }
   ],
   "source": [
    "input_ids = tokenizer.encode(\"terjemah Melayu ke pasar Melayu: rakyat memang tak suka awak pun\", return_tensors = 'pt')\n",
    "outputs = model.generate(input_ids, max_length = 100, do_sample=True, \n",
    "    top_k=50, \n",
    "    top_p=0.95, \n",
    "                         temperature=0.7,\n",
    "    num_return_sequences=3)\n",
    "print(tokenizer.batch_decode(outputs))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['Perbincangan khas juga bertujuan untuk Seri Paduka mendapat pandangan Raja2 Melayu bagi membolehkan Baginda membuat keputusan yang terbaik demi kepentingan dan kesejahteraan Negara serta Rakyat',\n",
       " 'Perbincangan khas juga bertujuan agar Baginda mendapat pandangan Raja2 Melayu bagi membolehkan baginda membuat keputusan yang terbaik demi kepentingan dan kesejahteraan negara dan rakyat',\n",
       " 'Perbincangan khas itu juga bertujuan utk baginda mendapat pandangan Raja2 Melayu utk membolehkan baginda membuat keputusan terbaik demi kepentingan dan kesejahteraan negara dan rakyat']"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "input_ids = tokenizer.encode(\"terjemah Melayu ke pasar Melayu: Perbincangan khas itu juga bertujuan bagi Seri Paduka mendapat pandangan Raja-Raja Melayu untuk membolehkan baginda membuat keputusan yang terbaik demi kepentingan dan kesejahteraan negara serta rakyat\", return_tensors = 'pt')\n",
    "outputs = model.generate(input_ids, max_length = 100, do_sample=True, \n",
    "    top_k=100, \n",
    "    top_p=0.95,\n",
    "                         temperature=0.7,\n",
    "    num_return_sequences=3)\n",
    "tokenizer.batch_decode(outputs, skip_special_tokens=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['Saya menulis untuk memohon jawatan Senior Software Engineer di [Company]. Sebagai seorang software engineer yang sangat mahir dan berpengalaman dengan kepakaran dalam seni bina data besar dan pemprosesan bahasa semulajadi, saya percaya saya akan menjadi tambahan yang berharga untuk pasukan anda',\n",
       " 'Saya menulis untuk memohon jawatan Senior Software Engineer di [Company]. Sebagai seorang jurutera perisian yang sangat mahir dan berpengalaman dengan kepakaran dalam seni bina data besar dan pemprosesan bahasa semulajadi, saya percaya saya akan menjadi penambahan yang berharga kepada pasukan anda',\n",
       " 'Saya menulis untuk memohon jawatan Senior Software Engineer di [Company]. Sebagai seorang software engineer yang sangat mahir dan berpengalaman dengan pengkhususan dalam seni bina data besar dan pemprosesan bahasa semulajadi, saya percaya saya akan menjadi penambahan berharga kepada pasukan anda']"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "input_ids = tokenizer.encode('terjemah Inggeris ke pasar Melayu: I am writing to apply for the Senior Software Engineer position at [Company]. As a highly skilled and experienced software engineer with a specialization in big data architecture and natural language processing, I believe I would be a valuable addition to your team', return_tensors = 'pt')\n",
    "outputs = model.generate(input_ids, max_length = 100, do_sample=True, \n",
    "    top_k=100, \n",
    "    top_p=0.95, temperature=0.7,\n",
    "    num_return_sequences=3)\n",
    "tokenizer.batch_decode(outputs, skip_special_tokens=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['nous', 'can', 'do', 'asr', 'not?']"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "t = 'nous bleh buat asr tak?'\n",
    "input_ids = [{'input_ids': tokenizer.encode(f'terjemah Melayu ke Inggeris: {s}', return_tensors='pt')[\n",
    "    0]} for s in t.split()]\n",
    "padded = tokenizer.pad(input_ids, padding='longest')\n",
    "outputs = model.generate(**padded, max_length = 50)\n",
    "tokenizer.batch_decode(outputs, skip_special_tokens=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<pad> asr</s>\n"
     ]
    }
   ],
   "source": [
    "input_ids = tokenizer.encode('terjemah Melayu ke Inggeris: asr', return_tensors = 'pt')\n",
    "outputs = model.generate(input_ids, max_length = 100)\n",
    "print(tokenizer.decode(outputs[0]))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<pad> Is there a promo?</s>\n"
     ]
    }
   ],
   "source": [
    "input_ids = tokenizer.encode('terjemah Melayu ke Inggeris: ada promo x?', return_tensors = 'pt')\n",
    "outputs = model.generate(input_ids, max_length = 100)\n",
    "print(tokenizer.decode(outputs[0]))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<pad> mesolitica can b</s>\n"
     ]
    }
   ],
   "source": [
    "input_ids = tokenizer.encode('terjemah Melayu ke Inggeris: mesolitica bleh b', return_tensors = 'pt')\n",
    "outputs = model.generate(input_ids, max_length = 100)\n",
    "print(tokenizer.decode(outputs[0]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['can', 'do', 'asr', 'not']"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "t = 'bleh buat asr tak'\n",
    "input_ids = [{'input_ids': tokenizer.encode(f'terjemah Melayu ke Inggeris: {s}', return_tensors='pt')[\n",
    "    0]} for s in t.split()]\n",
    "padded = tokenizer.pad(input_ids, padding='longest')\n",
    "outputs = model.generate(**padded, max_length = 50)\n",
    "tokenizer.batch_decode(outputs, skip_special_tokens=True)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<pad> can you make asr or not?</s>\n"
     ]
    }
   ],
   "source": [
    "input_ids = tokenizer.encode(f'terjemah Melayu ke Inggeris: {t}', return_tensors = 'pt')\n",
    "outputs = model.generate(input_ids, max_length = 100)\n",
    "print(tokenizer.decode(outputs[0]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[\"I don't understand\",\n",
       " 'Hi guys! I noticed yesterday & today many people have got these cookies, right? So today I want to share some post mortem of our first batch:',\n",
       " \"Indeed. This doesn't bother experts, I know too. It's a gesture, stupid.\",\n",
       " \"at 8 o'clock at the KK market it's really crowded, he's good at choosing a place.\",\n",
       " 'So haram jadah',\n",
       " 'where do you want to go?',\n",
       " \"It's like taking half a day\",\n",
       " \"Imagine PAKATAN HARAPAN and win pru-14. After that, there are all kinds of back doors. Ismail Sabri went up last. That's why I don't give a fk about politics anymore. I swear it's already up.\"]"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "strings = [\n",
    "    'ak tak paham la',\n",
    "    'Hi guys! I noticed semalam & harini dah ramai yang dapat cookies ni kan. So harini i nak share some post mortem of our first batch:',\n",
    "    \"Memanglah. Ini tak payah expert, aku pun tau. It's a gesture, bodoh.\",\n",
    "    'jam 8 di pasar KK memang org ramai 😂, pandai dia pilih tmpt.',\n",
    "    'Jadi haram jadah😀😃🤭',\n",
    "    'nak gi mana tuu',\n",
    "    'Macam nak ambil half day',\n",
    "    \"Bayangkan PH dan menang pru-14. Pastu macam-macam pintu belakang ada. Last-last Ismail Sabri naik. That's why I don't give a fk about politics anymore. Sumpah dah fk up dah.\",\n",
    "]\n",
    "input_ids = [{'input_ids': tokenizer.encode(f'terjemah Melayu ke Inggeris: {s}', return_tensors='pt')[\n",
    "    0]} for s in strings]\n",
    "padded = tokenizer.pad(input_ids, padding='longest')\n",
    "outputs = model.generate(**padded, max_length = 100)\n",
    "tokenizer.batch_decode(outputs, skip_special_tokens=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['Ini awak, cakap betul-betul',\n",
       " 'baru menghadiri majlis perkahwinan sepupu saya. Peliknya dia hanya mengadakan majlis biasa kerana gaya hidupnya kelihatan mewah. kemudian saya mendapat tahu mereka akan pergi pada bulan madu selama 3 minggu. keputusan yang bijak',\n",
       " 'Saya selepas melihat video ini: burger benjo extra mayo memang sedap',\n",
       " 'Hai kawan-kawan! Saya perasan semalam & hari ini ramai yang dapat biskut ni kan? Jadi hari ini saya ingin berkongsi beberapa post mortem batch pertama kami:']"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "strings = [\n",
    "    'u ni, talk properly lah',\n",
    "    \"just attended my cousin's wedding. pelik jugak dia buat majlis biasa2 je sebab her lifestyle looks lavish. then i found out they're going on a 3 weeks honeymoon. smart decision 👍\",\n",
    "    'Me after seeing this video: mm dapnya burger benjo extra mayo',\n",
    "    'Hi guys! I noticed semalam & harini dah ramai yang dapat cookies ni kan. So harini i nak share some post mortem of our first batch:',\n",
    "]\n",
    "input_ids = [{'input_ids': tokenizer.encode(f'terjemah pasar Melayu ke Melayu: {s}', return_tensors='pt')[\n",
    "    0]} for s in strings]\n",
    "padded = tokenizer.pad(input_ids, padding='longest')\n",
    "outputs = model.generate(**padded, max_length = 100)\n",
    "tokenizer.batch_decode(outputs, skip_special_tokens=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/husein/.local/lib/python3.8/site-packages/transformers/utils/hub.py:651: UserWarning: The `organization` argument is deprecated and will be removed in v5 of Transformers. Set your organization directly in the `repo_id` passed instead (`repo_id={organization}/{model_id}`).\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "CommitInfo(commit_url='https://huggingface.co/mesolitica/finetune-noisy-translation-t5-base-bahasa-cased-v2/commit/569bdfc042a26d6b4f35b7c0ce6cb977d5a799ac', commit_message='Upload T5ForConditionalGeneration', commit_description='', oid='569bdfc042a26d6b4f35b7c0ce6cb977d5a799ac', pr_url=None, pr_revision=None, pr_num=None)"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.push_to_hub('finetune-noisy-translation-t5-base-bahasa-cased-v2', organization='mesolitica')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "CommitInfo(commit_url='https://huggingface.co/mesolitica/finetune-noisy-translation-t5-base-bahasa-cased-v2/commit/565abb8debe26ead2a002c5ad8cf37c015f94f42', commit_message='Upload tokenizer', commit_description='', oid='565abb8debe26ead2a002c5ad8cf37c015f94f42', pr_url=None, pr_revision=None, pr_num=None)"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tokenizer.push_to_hub('finetune-noisy-translation-t5-base-bahasa-cased-v2', organization='mesolitica')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "_ = model.cuda()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "5000it [12:17,  6.78it/s]\n"
     ]
    }
   ],
   "source": [
    "from tqdm import tqdm\n",
    "import json\n",
    "\n",
    "filtered_left, filtered_right = [], []\n",
    "\n",
    "with open('shuffled-test.json') as fopen:\n",
    "    for l in tqdm(fopen):\n",
    "        data = json.loads(l)['translation']\n",
    "        p = data['prefix']\n",
    "        src = data['src']\n",
    "        input_ids = [{'input_ids': tokenizer.encode(f'{p}: {src}', return_tensors = 'pt')[0]}]\n",
    "        padded = tokenizer.pad(input_ids, padding = 'longest')\n",
    "        for k in padded.keys():\n",
    "            padded[k] = padded[k].cuda()\n",
    "        outputs = model.generate(**padded, max_length = 256)\n",
    "        filtered_left.append(tokenizer.decode(outputs[0], skip_special_tokens=True))\n",
    "        filtered_right.append(data['tgt'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "refs = [filtered_right]\n",
    "sys = filtered_left"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sacrebleu.metrics import BLEU, CHRF, TER\n",
    "\n",
    "bleu = BLEU()\n",
    "chrf = CHRF(word_order = 2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'name': 'BLEU',\n",
       " 'score': 64.583819005204,\n",
       " '_mean': -1.0,\n",
       " '_ci': -1.0,\n",
       " '_verbose': '80.2/68.1/59.8/53.2 (BP = 1.000 ratio = 1.048 hyp_len = 112260 ref_len = 107150)',\n",
       " 'bp': 1.0,\n",
       " 'counts': [90014, 73084, 61157, 51798],\n",
       " 'totals': [112260, 107260, 102260, 97281],\n",
       " 'sys_len': 112260,\n",
       " 'ref_len': 107150,\n",
       " 'precisions': [80.1835025832888,\n",
       "  68.13723662129405,\n",
       "  59.805398005085074,\n",
       "  53.2457519967928],\n",
       " 'prec_str': '80.2/68.1/59.8/53.2',\n",
       " 'ratio': 1.047690153989734}"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "r = bleu.corpus_score(sys, refs)\n",
    "r.__dict__"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
